% scribe: YiFeng Li
% lastupdate: Oct. 10, 2005
% lecture: 10
% title: Kolmogorov's Law of Large Numbers
% references: Durrett, section 1.8
% keywords: Kolmogorov's Law of Large Numbers, law of the iterated logarithm, convergence in distribution, weak convergence
% end

\documentclass[12pt,letterpaper]{article}

\include{macros}

\newcommand{\convpoint}{\stackrel{p.w.}{\longrightarrow}}
\newcommand{\conv}{\rightarrow}

\newtheorem{fact}[theorem]{Fact}

\begin{document}


\lecture{10}{Kolmogorov's Law of Large Numbers}{YiFeng
Li}{yfli@berkeley.edu}

(These notes are a revision of the work of Vinod Prabhakaran,
2002.)


\section{Law of the Iterated Logarithm}
% keywords: law of the iterated logarithm
% end

Let $X_{1}$, $X_{2}$, ... be i.i.d. with $\E X_{i}=0$, $\E X_{i}^{2}=\sigma ^{2}$%
, $S_{n}=X_{1}+...+X_{n}.$ We know
\[
\frac{S_{n}}{n^{\frac{1}{2}+\varepsilon
}}\overset{a.s}{\longrightarrow }0\text{ \ as }n\rightarrow \infty
.
\]We will show later%
\[
\frac{S_{n}}{\sigma n^{\frac{1}{2}}}\overset{d}{\longrightarrow
}N(0,1)\text{\ as }n\rightarrow \infty .
\]
For general interest, we state, without proof, the \emph{Law of the Iterated 
Logarithm}:

\[
\underset{n\rightarrow \infty }{\lim \sup }\frac{S_{n}}{\sigma
\sqrt{2n\log (\log n)}}=1\text{ a.s.}
\]
\[
\underset{n\rightarrow \infty }{\lim \inf }\frac{S_{n}}{\sigma
\sqrt{2n\log (\log n)}}=1\text{ a.s.}
\]\[
\P(S_{n}>(1+\varepsilon )\sigma \sqrt{2n\log (\log n)}\text{
i.o.})=0
\]%
\[
\P(S_{n}>(1-\varepsilon )\sigma \sqrt{2n\log (\log n)}\text{
i.o.})=0
\]
\section{Kolmogorov's Law of Large Numbers}
% keywords: Kolmogorov's Law of Large Numbers
% end

\begin{theorem}
Let $X_{1}$, $X_{2}$, ... be i.i.d. with $\E\!\left( \left\vert
X_{i}\right\vert \right) <\infty $, $S_{n}=X_{1}+...+X_{n}.$ Then $%
S_{n}/n\rightarrow \E\!\left( X\right) $ a.s. as $n\rightarrow \infty
.$
\end{theorem}

Note that the theorem is true with just pairwise independence instead
of the full independence assumed here [\cite{durrett}, p.55 (7.1)]. The
theorem also has an important generaliztion
to stationary sequences (the \emph{ergodic theorem}, [\cite{durrett},
p.337 (2.1)]).


\begin{proof}
\textit{Step 1}: Replace $X_{i}$ by $\widetilde{X}_{i}=X_{i}-\E X$ 
(note $\E X_{i}=\E X$). Then

\begin{equation*}
\frac{\widetilde{S}_{n}}{n}=\frac{S_{n}}{n}-\E X.
\end{equation*}%
So it's enough to consider $\E X=0.$

\textit{Step 2}: Now we assume $\E X=0$.  Introduce truncated variables%
\begin{equation*}
\widehat{X}_{n}:=X_{n}I\left( \left\vert X_{n}\right\vert \leq
n\right) .
\end{equation*}%
Observe that%
\begin{equation*}
\P(X_{n}=\widehat{X}_{n}\text{ }ev\text{.})=1.
\end{equation*}%
(To see this, check%
\begin{equation*}
\P(X_{n}\neq \widehat{X}_{n}\text{ }i.o\text{.})=\P\!\left( \left\vert
X_{n}\right\vert >n\text{ }i.o\text{.}\right)
\end{equation*}%
\begin{equation*}
\sum\limits_{n=1}^{\infty }\P\!\left( \left\vert X_{n}\right\vert >n\text{ }%
\right) =\sum\limits_{n=1}^{\infty }\P\!\left( \left\vert
X\right\vert >n\text{ }\right) =\E\!\left( \sum\limits_{n=1}^{\infty
}I\left( \left\vert X\right\vert >n\right) \right) <\infty
\end{equation*}%
since
\begin{equation*}
\sum\limits_{n=1}^{\infty }I\left( \left\vert X\right\vert
>n\right) =\sum\limits_{1\leq n<X}1\leq \left\vert x\right\vert
+1.
\end{equation*} 
Compare this to the tail sum formula for a random variable$X$ with values in $0,1,2,...$%
\begin{equation*}
\E X=\sum\limits_{n=1}^{\infty }n\P\!\left( X=n\right)
=\sum\limits_{n=1}^{\infty }\P\!\left( X\geq n\right) .
\end{equation*}

\textit{Step 3: }Center the truncated variables. Define $\widetilde{X}_{n}:=%
\widehat{X}_{n}-\E\!\left( \widehat{X}_{n}\right) .$\\We will show
that\[ \left(\frac{S_n}{n}\rightarrow0\right)
\stackrel{a.s.}{\stackrel{=}{\mbox{\tiny (a)}}}
\left(\frac{\hat{S}_n}{n}\rightarrow0\right)
\stackrel{a.s.}{\stackrel{=}{\mbox{\tiny (b)}}}
\left(\frac{\tilde{S}_n}{n}\rightarrow0\right),\] where
$\hat{S}_n=\hat{X}_1+\hat{X}_2+\cdots+\hat{X}_n$ and
$\tilde{S}_n=\tilde{X}_1+\tilde{X}_2+\cdots+\tilde{X}_n$.  Then
using Kronecker's lemma we will show that
$\P\!\left({\tilde{S}_n}/{n} \rightarrow0\right)=1$.

(a) comes from the fact that if $\omega \in \left\{ \omega :X_{n}\left( \omega \right) =\widehat{X}%
_{n}\left( \omega \right) \text{ }ev.\right\}$ (which has probablity 1), then $%
S_{n}\left( \omega \right) -\widehat{S}_{n}\left( \omega \right) $
is eventually not dependent on $n$. So
\begin{equation*}
\frac{S_{n}\left( \omega \right) -\widehat{S}_{n}\left( \omega \right) }{n}%
\rightarrow 0\text{ \ for such }\omega.
\end{equation*}%

(b) comes from
\begin{equation*}
\frac{\widehat{S}_{n}}{n}-\frac{\widetilde{S}_{n}}{n}=\frac{\E\widehat{X}%
_{1}+\E\widehat{X}_{2}+...+\E\widehat{X}_{n}}{n}\rightarrow 0\text{ as }%
n\rightarrow \infty \text{ \ (By analysis and }\E\widehat{X}_{i}\rightarrow 0%
\text{)}
\end{equation*}
%
But
%
\begin{equation*}
\E\widehat{X}_{n}=\E\!\left[ X_{n}I\left( \left\vert X_{n}\right\vert
\leq
n\right) \right] =\E\!\left[ XI\left( \left\vert X\right\vert \leq n\right) %
\right] \rightarrow \E X\text{ as }n\rightarrow \infty .
\end{equation*}%
as the integrand is dominated by $\left\vert X\right\vert$ and
note $\\E\!\left( \left\vert X\right\vert \right) <\infty$.

Now, we use Kronecker's lemma and the ${\cal L}^2$ convergence theorem to show that%
\begin{equation*}
\sum_{n=1}^{\infty }\frac{\E\!\left( \widetilde{X}_{n}^{2}\right) }{n^{2}}%
<\infty .
\end{equation*}%
\begin{equation*}
\E\!\left( \widetilde{X}_{n}^{2}\right) =\E\!\left[ \left(
\widehat{X}_{n} -\E\!\left( \widehat{X}_{n}\right) \right)^{2}\right]
=\E\!\left[ \left( X\1\left( \left\vert X\right\vert \leq n\right)
 -\E\!\left( X\1\left( \left\vert X\right\vert \leq n\right)
\right)\right)^{2}\right]
\end{equation*}%
\begin{equation*}
\leq \E\!\left( X\1\left( \left\vert X\right\vert \leq n\right)
\right) ^{2}.
\end{equation*}%
So%
\begin{equation*}
\sum_{n=1}^{\infty }\frac{\E\!\left( \widetilde{X}_{n}^{2}\right)
}{n^{2}}\leq \sum_{n=1}^{\infty }\frac{\E X^{2}I\left( \left\vert
X\right\vert \leq n\right) }{n^{2}}=\E\!\left(
X^{2}\sum_{n=1}^{\infty }\frac{I\left( \left\vert X\right\vert
\leq n\right) }{n^{2}}\right)
\end{equation*}%
\begin{equation*}
\approxeq \E\!\left( \frac{X^{2}}{\left\vert X\right\vert }\right)
=\E\!\left( \left\vert X\right\vert \right) <\infty .
\end{equation*}%

This came from
%
\begin{equation*}
\sum_{n=1}^{\infty }\frac{x^{2}1_{\left( \left\vert x\right\vert
\leq
n\right) }}{n^{2}}\approxeq x^{2}\sum_{n=1}^{\infty }\frac{1}{n^{2}}%
\approxeq x^{2}\frac{1}{\left\vert x\right\vert}
\approxeq \left\vert x\right\vert .
\end{equation*}
\end{proof}


\section{Convergence in distribution}
% keywords: convergence in distribution, weak convergence
% end

\begin{definition}
$X_{n}\overset{d}{\rightarrow }X$ if $\P\!\left( X_{n}\leq x\right)
\rightarrow \P\!\left( X\leq x\right) $ for all $x$ at which
$x\rightarrow \P\!\left( X\leq x\right) $ is continuous.  We call this
\emph{convergence in distribution} or \emph{weak convergence}.
\end{definition}

\textit{Note. }This is really a notion of convergence of
probability measures rather than of convergence of random
variables. Now the limit random
variable $X$ is only unique in distribution, not unique almost surely.  
Obviously, if $X%
\overset{d}{=}Y$ and $X_{n}\overset{d}{\rightarrow }X,$ then $X_{n}\overset{d%
}{\rightarrow }Y$.


\begin{theorem}[Skorokhod]
$X_{n}\overset{d}{\rightarrow }X\Longleftrightarrow $ there exists a
probability with
space random variables $Y_{n}$ with $Y_{n}\overset{d}{=}X_{n},$ $Y%
\overset{d}{=}X$ and $Y_{n}\overset{a.s.}{\rightarrow }Y$.
\end{theorem}

\begin{proof}
Take a single uniform variable $U$ and use it to create the $Y_{n}$
and $Y$.  Let
\begin{equation*}
F_{n}\!\left( x\right) =\P\!\left( X_{n}\leq x\right) ,
\end{equation*}%
\begin{equation*}
F\!\left( x\right) =\P\!\left( X\leq x\right) ,
\end{equation*}%
\begin{equation*}
Y_{n}=F_{n}^{-1}(U),
\end{equation*}%
\begin{equation*}
Y=F^{-1}(U).
\end{equation*}%
Check that
\begin{equation*}
F^{-1}(U)=\inf \left\{ x:f\!\left( x\right) >u\right\} .
\end{equation*}
\end{proof}

The following proposition is an application of the above.

\begin{proposition}
If $X_{n}\overset{d}{\rightarrow }X$, then for every bounded
continuous
function $f:R\rightarrow R$%
\begin{equation*}
\E\!\left[ f\!\left( X_{n}\right) \right] \rightarrow \E\!\left[ f\!\left( X\right) %
\right] .
\end{equation*}
\end{proposition}

\begin{proof}
Without loss of generality, $X_{n}\overset{a.s.}{\rightarrow }X$.  Then $%
f\!\left( X_{n}\right) \rightarrow f\!\left( X\right) $ is bounded, so we can
take expectations and use the bounded convergence theorem.
\end{proof}

%\begin{thebibliography}{9}
%\bibitem{durrett} R. Durrett. \textit{Probability: Theory and Examples}. Duxbury
%Press, Belmont, CA, Third edition, 2005.
%\end{thebibliography}
\bibliographystyle{plain}
\bibliography{../books}

\end{document}